package main

import (
	//"bufio"
	"bytes"
	"code.google.com/p/go.text/encoding/simplifiedchinese"
	"code.google.com/p/go.text/transform"
	"flag"
	"fmt"
	goq "github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	main_domain = "" //主域名
	waitgroup   = &sync.WaitGroup{}
	LOG         = "fang"
)

func main() {
	var citycode = flag.String("citycode", "bj", "请输入城市编号") //城市编号
	flag.Parse()
	switch *citycode {
	case "bj":
		{
			main_domain = "http://esf.fang.com"
			break
		}
	default:
		{
			main_domain = "http://esf." + *citycode + ".fang.com"
			break
		}
	}
	LOG = *citycode + "." + LOG

	tmpurl := fmt.Sprintf("%s/agenthome/", main_domain)
	doc, e := getDoc(tmpurl)
	if e != nil {
		fmt.Println(e.Error())
		os.Exit(-1)
	}
	doc.Find(".qxName").Children().Not(".orange").Each(func(i int, sec *goq.Selection) {
		var (
			qx     string //区县
			qxlink string //区县链接
		)
		qxlink, ok := sec.Attr("href")
		if ok {
			qx = filterStr(sec.Text())
			getBlock(qxlink, qx)
		}
	})

	fmt.Println("done.")
}

//获取热点
func getBlock(url, district string) error {
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		return e
	}
	doc.Find("#shangQuancontain").Children().Not(".orange").Each(func(i int, sec *goq.Selection) {
		var (
			block     string //热点
			blocklink string //热点链接
		)
		blocklink, ok := sec.Attr("href")
		if ok {
			block = filterStr(sec.Text())
			getBlockSubpage(blocklink, block, district)
		}
	})
	return nil
}

//获取热点分页
func getBlockSubpage(url, block, district string) error {
	tmpurl := strings.Replace(url, "-i31-j310/", "", 1)
	homeurl := fmt.Sprintf("%s%s-i3%d-j310/", main_domain, tmpurl, 1) //首页
	doc, e := getDoc(homeurl)
	if e != nil {
		return e
	}

	page := doc.Find(".fy_text").Text()
	page = strings.Replace(page, "1/", "", 1)
	pageCount, e := strconv.Atoi(page)
	if e != nil {
		s(homeurl + " 找不到页码")
		return e
	}

	var NumCPU = 2
	runtime.GOMAXPROCS(NumCPU)
	for curpage := 1; curpage <= pageCount; curpage += NumCPU {
		for i := curpage; i < curpage+NumCPU && i <= pageCount; i++ {
			waitgroup.Add(1)
			suburl := fmt.Sprintf("%s-i3%d-j310/", tmpurl, i)
			var stdout = fmt.Sprintf("(%d/%d) %s %s %s", i, pageCount, G2U(district), G2U(block), G2U(suburl))
			fmt.Fprintln(os.Stdout, stdout)
			go getAgent(suburl, block, district)
		}
		waitgroup.Wait()
		time.Sleep(time.Second * 10)
	}
	return nil
}

//获取经纪人
func getAgent(url, block, district string) {
	defer waitgroup.Done()
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		return
	}

	var retMap = []map[string]string{}
	doc.Find(".agent_pic").Children().Each(func(i int, sec *goq.Selection) {
		var (
			name          string //经纪人改名
			companyname   string //经纪人所属公司
			telphone      string //联系电话
			shangQuan     string //服务商圈
			shopName      string //店铺名称
			housenum      string //房源数
			communitylist string //服务楼盘
		)

		shoplink, _ := sec.Find(".pic").Find("a").Eq(0).Attr("href")
		shoplink = filterStr(shoplink)

		name = filterStr(sec.Find(".housetitle").Find("a").Eq(0).Text())
		name = G2U(name)
		if name == "" {
			name = "--"
		}

		kw := U2G("所属公司")
		companyname = sec.Find(".black:contains('" + kw + "')").Find("span").Text()
		companyname = filterStr(companyname)
		companyname = G2U(companyname)
		if companyname == "" {
			companyname = "--"
		}

		kw = U2G("联系电话")
		telphone = sec.Find(".black:contains('" + kw + "')").Find("strong").Text()
		telphone = filterStr(telphone)
		if telphone == "" {
			telphone = "--"
		}

		shangQuan = ""
		kw = U2G("服务商圈")
		sec.Find(".blue:contains('" + kw + "')").Find("a").Each(func(i2 int, sec2 *goq.Selection) {
			shangQuan += "," + sec2.Text()
		})
		shangQuan = strings.Trim(shangQuan, ",")
		shangQuan = G2U(shangQuan)
		if shangQuan == "" {
			shangQuan = "--"
		}

		shopName, housenum, communitylist = getShopInfo(shoplink)
		shopName = G2U(shopName)
		if shopName == "" {
			shopName = "--"
		}
		communitylist = G2U(communitylist)
		if communitylist == "" {
			communitylist = "--"
		}

		newmap := map[string]string{
			"name":          name,
			"companyname":   companyname,
			"telphone":      telphone,
			"shangQuan":     shangQuan,
			"shopName":      shopName,
			"housenum":      housenum,
			"communitylist": communitylist,
		}
		retMap = append(retMap, newmap)
	})

	for _, v := range retMap {
		str := G2U(district)
		str += "||" + G2U(block)
		str += "||" + v["name"]
		str += "||" + v["companyname"]
		str += "||" + v["shopName"]
		str += "||" + v["housenum"]
		str += "||" + v["telphone"]
		str += "||" + v["shangQuan"]
		str += "||" + v["communitylist"]
		//fmt.Println(str)
		outlog(str)
	}
}

//获取店铺信息
func getShopInfo(url string) (string, string, string) {
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		return "", "", ""
	}
	kw := U2G("门店名称：")
	tmptxt := doc.Find("li:contains('" + kw + "')").Text()
	//店铺名称
	shopName := filterStr(strings.Replace(tmptxt, kw, "", 1))

	//房源数
	housenum := 0
	doc.Find(".searchlist").Find("strong").Each(func(i int, sec *goq.Selection) {
		var tmptxt = sec.Text()
		kw := U2G("（")
		tmparr := strings.Split(tmptxt, kw)
		if len(tmparr) > 1 {
			tmptxt = strings.Replace(tmparr[1], U2G("）"), "", 1)
		} else {
			tmptxt = "0"
		}
		tmpnum, _ := strconv.Atoi(tmptxt)
		housenum += tmpnum
	})
	//服务楼盘
	communitylist := ""
	map_communitylist := make(map[string]string)
	if housenum != 0 {
		doc.Find(".searchlist").Find("li").Find("a").Each(func(i int, sec *goq.Selection) {
			if title, ok := sec.Attr("title"); ok {
				title = filterStr(title)
				map_communitylist[title] = title
			}
		})
	}
	if len(map_communitylist) > 0 {
		arr_communitylist := []string{}
		for _, v := range map_communitylist {
			arr_communitylist = append(arr_communitylist, v)
		}
		communitylist = strings.Join(arr_communitylist, ",")
	}
	return shopName, strconv.Itoa(housenum), communitylist
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := getResp(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

//获取response
func getResp(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	//req.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Add("Cache-Control", "max-age=0")
	req.Header.Add("Proxy-Connection", "keep-alive")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(25 * time.Second)
				c, err := net.DialTimeout(netw, addr, time.Second*20)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	var resp *http.Response
	//尝试三次
	for retry := 0; retry < 3; retry++ {
		resp, e = c.Do(req)
		if e != nil {
			time.Sleep(time.Second * 1)
		} else {
			break
		}
	}
	if e != nil {
		return nil, e
	}
	return resp, nil
}

//gbk->utf8
func G2U(s string) string {
	ret, e := gbktoutf8([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

//utf8->gbk
func U2G(s string) string {
	ret, e := utf8togbk([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

func gbktoutf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}
func utf8togbk(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "||", "", -1)
	str = strings.Replace(str, ",", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
